from xrs import get_xrs_id_list, get_xrs_question
from bson.objectid import ObjectId
import requests
from bs4 import BeautifulSoup
import re
from mongo import MongoConn
import time
import queue
import threading
import logging


def get_logger(filename):
    logger = logging.getLogger()
    logger.setLevel(logging.WARNING)
    fh = logging.FileHandler(filename)
    fh.setLevel(logging.WARNING)
    ch = logging.StreamHandler()
    ch.setLevel(logging.WARNING)
    formatter = logging.Formatter('[%(asctime)s][%(filename)s][%(levelname)s] ## %(message)s')
    fh.setFormatter(formatter)
    ch.setFormatter(formatter)
    logger.addHandler(fh)
    logger.addHandler(ch)
    logger.info('start')
    return logger


if __name__ == '__main__':
    logger = get_logger('logs/high.log')
    q = queue.Queue()
    for i in range(1, 5548):
        q.put(i)

    lock = threading.Lock()

    def parse(thread_num):
        url_pattern = 'http://tiku.xueersi.com/shiti/list_3_1_0_0_4_0_{}'
        m = MongoConn()
        while q.empty() is False:
            i = q.get()
            ids = get_xrs_id_list(url_pattern.format(i))
            lock.acquire()
            logger.warning('thread {} parse page {}'.format(thread_num, i))
            lock.release()
            for id_ in ids:
                try:
                    question_json = get_xrs_question(id_)
                    time.sleep(0.5)
                    question_json['source'] = 'xrs'
                    question_json['difficulty'] = 'high'
                    question_json['page'] = i
                    m.insert_one(question_json)
                except Exception as e:
                    lock.acquire()
                    logger.warning("{} {}".format(str(e), str(id_)))
                    lock.release()

    threads = []
    for i in range(10):
        threads.append(threading.Thread(target=parse, args=(i,)))

    for t in threads:
        t.setDaemon(True)
        t.start()

    for t in threads:
        t.join()
